add Intel TBB

Jenkinsfile

@@ -14,6 +14,14 @@ def runTests(String prefix = "") {
     """
 }
 
+def runWinTests(String prefix = "") {
+    withEnv(['PATH+TBB=./stan/lib/stan_math/lib/tbb']) {
+       """ mingw32-make -j${env.PARALLEL} build
+         ${prefix}runCmdStanTests.py -j${env.PARALLEL} src/test/interface
+       """
+    }
+}
+
 def deleteDirWin() {
     bat "attrib -r -s /s /d"
     deleteDir()
@@ -58,7 +66,7 @@ pipeline {
                     agent { label 'windows' }
                     steps {
                         setupCXX()
-                        bat runTests()
+                        bat runWinTests()
                     }
                     post {
                         always {
@@ -86,6 +94,7 @@ pipeline {
                     steps {
                         setupCXX("${MPICXX}")
                         sh "echo STAN_MPI=true >> make/local"
+                        sh "echo CXX_TYPE=gcc >> make/local"
                         sh "make build-mpi > build-mpi.log 2>&1"
                         sh runTests("./")
                     }

install-tbb.bat

@@ -0,0 +1,9 @@
+@ECHO off
+echo Permanently setting the PATH user environment variable:
+
+for /F "tokens=2* delims= " %%f IN ('reg query HKCU\Environment /v PATH ^| findstr /i path') do set OLD_SYSTEM_PATH=%%g
+setx Path %~dp0stan\lib\stan_math\lib\tbb;%OLD_SYSTEM_PATH%
+
+echo Please close this shell and open a new shell.
+echo This will make the changes to the PATH variable
+echo become active.

make/program

@@ -34,11 +34,11 @@ endif
 
 
 .PRECIOUS: %.hpp
-%$(EXE) : %.hpp $(CMDSTAN_MAIN_O) $(LIBSUNDIALS) $(MPI_TARGETS)
+%$(EXE) : %.hpp $(CMDSTAN_MAIN_O) $(LIBSUNDIALS) $(MPI_TARGETS) $(TBB_TARGETS)
 	@echo ''
 	@echo '--- Compiling, linking C++ code ---'
 	$(COMPILE.cpp) $(CXXFLAGS_PROGRAM) -x c++ -o $(subst  \,/,$*).o $(subst \,/,$<)
-	$(LINK.cpp) $(subst \,/,$*.o) $(CMDSTAN_MAIN_O) $(LDLIBS) $(LIBSUNDIALS) $(MPI_TARGETS) $(subst \,/,$(OUTPUT_OPTION))
+	$(LINK.cpp) $(subst \,/,$*.o) $(CMDSTAN_MAIN_O) $(LDLIBS) $(LIBSUNDIALS) $(MPI_TARGETS) $(TBB_TARGETS) $(subst \,/,$(OUTPUT_OPTION))
 
 ##
 # Dependencies file

make/tests

@@ -5,7 +5,7 @@
 test/%$(EXE) : CXXFLAGS += $(CXXFLAGS_GTEST)
 test/%$(EXE) : CPPFLAGS += $(CPPFLAGS_GTEST)
 test/%$(EXE) : INC += $(INC_GTEST) -I $(RAPIDJSON)
-test/%$(EXE) : test/%.o $(GTEST)/src/gtest_main.cc $(GTEST)/src/gtest-all.o $(LIBSUNDIALS) $(MPI_TARGETS)
+test/%$(EXE) : test/%.o $(GTEST)/src/gtest_main.cc $(GTEST)/src/gtest-all.o $(LIBSUNDIALS) $(MPI_TARGETS) $(TBB_TARGETS)
 	$(LINK.cpp) $(filter-out src/test/test-models/% src/%.csv bin/% test/%.hpp %.hpp-test,$^) $(LDLIBS) $(OUTPUT_OPTION)
 
 test/%.o : src/test/%.cpp

makefile

@@ -152,14 +152,28 @@ build-mpi: $(MPI_TARGETS)
 	@echo '--- boost mpi bindings built ---'
 
 .PHONY: build
-build: bin/stanc$(EXE) bin/stansummary$(EXE) bin/print$(EXE) bin/diagnose$(EXE) $(LIBSUNDIALS) $(MPI_TARGETS) $(CMDSTAN_MAIN_O)
+build: bin/stanc$(EXE) bin/stansummary$(EXE) bin/print$(EXE) bin/diagnose$(EXE) $(LIBSUNDIALS) $(MPI_TARGETS) $(TBB_TARGETS) $(CMDSTAN_MAIN_O)
 	@echo ''
+ifeq ($(OS),Windows_NT)
+		@echo 'NOTE: Please add $(TBB_BIN_ABSOLUTE_PATH) to your PATH variable.'
+		@echo 'You may call'
+		@echo ''
+		@echo 'mingw32-make install-tbb'
+		@echo ''
+		@echo 'to automatically update your user configuration.'
+endif
 	@echo '--- CmdStan v$(CMDSTAN_VERSION) built ---'
 
 ifeq ($(CXX_TYPE),clang)
 build: $(STAN)src/stan/model/model_header.hpp.gch
 endif
 
+.PHONY: install-tbb
+install-tbb: $(TBB_TARGETS)
+ifeq ($(OS),Windows_NT)
+	$(shell echo "cmd.exe /C install-tbb.bat")
+endif
+
 ##
 # Clean up.
 ##
@@ -220,7 +234,7 @@ manual: src/docs/cmdstan-guide/cmdstan-guide.pdf
 
 .PHONY: compile_info
 compile_info:
-	@echo '$(LINK.cpp) $(CXXFLAGS_PROGRAM) $(CMDSTAN_MAIN_O) $(LDLIBS) $(LIBSUNDIALS) $(MPI_TARGETS)'
+	@echo '$(LINK.cpp) $(CXXFLAGS_PROGRAM) $(CMDSTAN_MAIN_O) $(LDLIBS) $(LIBSUNDIALS) $(MPI_TARGETS) $(TBB_TARGETS)'
 
 ##
 # Debug target that allows you to print a variable

src/docs/cmdstan-guide/compiling.tex

@@ -80,7 +80,7 @@ \subsection{Getting Help from the {\tt makefile}}
 \subsection{Options to \code{make}}
 
 \CmdStan allows users to change compilers, library versions for Boost,
-Eigen, and CVODES, as well as compilation options such as
+Eigen, Sundials, and Intel TBB, as well as compilation options such as
 optimization.
 
 For a full list of options, see \refappendix{make-options}

src/docs/cmdstan-guide/getting-started.tex

@@ -15,7 +15,11 @@ \section{Installation}
   download.
 \item The \code{make} utility program. \\
   This is not strictly necessary, but will make the build process
-  easy. The rest of the documentation assumes \code{make} is available.
+  easy. On Windows the \code{mingw32-make} variant is needed to build
+  \CmdStan. This is a \code{make} variant and can be installed as part
+  of RTools (\url{https://cran.rstudio.com/bin/windows/Rtools/}). The
+  rest of the documentation assumes \code{make} (\code{mingw32-make}
+  on Windows) is available.
 \item A \Cpp compiler.
 \end{itemize}
 
@@ -89,7 +93,25 @@ \section{Building \CmdStan}
     \end{quote}
     %
     \textbf{Warning:} \ The \code{make} program may take 10+ minutes and
-    consume 2+ GB of memory to build \CmdStan.
+    consume 2+ GB of memory to build \CmdStan. Please use
+    \code{mingw32-make} on Windows (available from RTools).
+  \item \emph{Windows only:} \, \CmdStan requires that the Intel TBB
+    library, which is build by the above command, can be found by the
+    Windows system. This requires that the directory
+    \verb+<cmdstan-home>/stan/lib/stan_math/lib/tbb+ is part of the
+    \code{PATH} environment variable. To permanently make this setting
+    for the current user, you may execute:
+    %
+    \begin{quote}
+      \begin{Verbatim}[fontshape=sl]
+> mingw32-make install-tbb
+      \end{Verbatim}
+    \end{quote}
+    %
+    Don't forget to open a new shell where these new settings will
+    take effect. For other platforms this step is not needed, since
+    the absolute path to the Intel TBB library is linked into Stan
+    programs (not possible on Windows).
 \end{enumerate}
 %
 When \CmdStan is successfully built, the \code{make} program will

src/docs/cmdstan-guide/installation.tex

@@ -39,8 +39,8 @@ \subsection{\CmdStan Source}
 required.  The \CmdStan source code distribution includes \CmdStan's
 source code, Stan's source code, documentation, build scripts, unit
 tests, documentation and source for the required libraries, Stan Math
-Library, Boost, and Eigen, and the source for an optional testing
-library, Google Test.
+Library, Boost, Eigen, Sundials, Intel TBB, and the source for an
+optional testing library, Google Test.
 
 \subsubsection{CmdStan: Stable Releases}
 
@@ -116,6 +116,31 @@ \subsubsection{Eigen Matrix and Linear Algebra Library Source}
 %
 The Eigen source code is distributed with \CmdStan.
 
+\subsubsection{SUite of Nonlinear and DIfferential/ALgebraic Equation Solvers}
+
+Some of Stan's ordinary differenetial equation (ODE) solvers and
+implicit function solvers use the Sundials library.
+%
+\begin{itemize}
+\item Home: \url{https://computing.llnl.gov/projects/sundials}
+\item License: BSD
+\item Tested Version: 4.1.0
+\end{itemize}
+%
+The Sundials source code is distributed with \CmdStan.
+
+\subsubsection{Intel Threading Building Blocks}
+
+Stan uses the Intel TBB library for advanced threading support.
+%
+\begin{itemize}
+\item Home: \url{https://software.intel.com/en-us/tbb}
+\item License: Apache 2.0
+\item Tested Version: 2019 update 8
+\end{itemize}
+%
+The Intel TBB source code is distributed with \CmdStan.
+
 
 \subsection{\Cpp Compiler}
 
@@ -204,10 +229,10 @@ \subsection{Verifying Tools}
 \end{quote}
 %
 This should report version information for \gpp.  Next, verify that
-\code{make} is installed with the following command.
+make variant \code{mingw32-make} is installed with the following command.
 %
 \begin{quote}
-  \Verb|> make -v|
+  \Verb|> mingw32-make -v|
 \end{quote}
 %
 This should print version information for make.
@@ -484,10 +509,10 @@ \subsubsection{{\tt stanc} Options}
 \subsubsection{Changing Library Locations}
 
 The next set of variables allow for easy replacement of dependent
-libraries. \CmdStan depends on Stan, Boost, Eigen, and
-CVODES. Although \CmdStan is bundled with a particular verison of
+libraries. \CmdStan depends on Stan, Boost, Eigen, Sundials, and Intel
+TBB. Although \CmdStan is bundled with a particular verison of
 Stan, other versions can be used. The same is true with Boost, Eigen,
-and CVODES.
+Sundials, and Intel TBB.
 
 The Stan developers typically replace the tagged version of Stan in
 \code{<cmdstan-home>/stan} with an updated version and do not set
@@ -502,13 +527,16 @@ \subsubsection{Changing Library Locations}
     default is \Verb|MATH=stan/l../stan_math_2.20.0/|.
   \item \Verb|BOOST|. The location of the Boost library. If
     \Verb|BOOST| is not explicitly set, this will default to the
-    \Verb|lib/boost_1.66.0/| folder within \Verb|MATH|.
+    \Verb|lib/boost_1.69.0/| folder within \Verb|MATH|.
   \item \Verb|EIGEN|. The location of the Eigen library. If
     \Verb|EIGEN| is not explicitly set, this will default to the
     \Verb|lib/eigen_3.3.3/| folder within \Verb|MATH|.
   \item \Verb|SUNDIALS|. The location of the SUNDIALS library. If
     \Verb|SUNDIALS| is not explicity set, this will default to the
-    \Verb|lib/sundials_3.1.0/| folder with \Verb|MATH|.
+    \Verb|lib/sundials_4.1.0/| folder within \Verb|MATH|.
+  \item \Verb|TBB|. The location of the Intel TBB library. If
+    \Verb|TBB| is not explicity set, this will default to the
+    \Verb|lib/tbb_2019_U8| folder within \Verb|MATH|.
 \end{itemize}
 %
 
@@ -644,6 +672,9 @@ \subsection{GNU Make Build Tool}
 \item Tested Versions: 3.81 (Mac OS X), 3.79 (Windows 7)
 \end{itemize}
 %
+On Windows \code{mingw32-make} variant must be used as a requirement
+to build the Intel TBB. This variant of \code{make} is available as
+part of RTools \url{https://cran.rstudio.com/bin/windows/Rtools/}.
 
 
 \subsection{Doxygen Documentation Generator}

src/docs/cmdstan-guide/introduction.tex

@@ -21,8 +21,8 @@ \section{Licensing}
 
 CmdStan, Stan, and the Stan Math Library are licensed under the new
 BSD license (3-clause).  See \refappendix{licensing} for details,
-including licensing terms for the dependent packages Boost, Eigen, and
-CVODES.
+including licensing terms for the dependent packages Boost, Eigen,
+Sundials, and Intel TBB.
 
 
 \section{Modeling Language User's Guide and Reference}

src/docs/cmdstan-guide/licensing.tex

@@ -1,8 +1,9 @@
 \chapter{Licensing}\label{licensing.appendix}
 
 \noindent
-\CmdStan, Stan, and Stan's three dependent libraries, Stan Math Library, Boost, and Eigen, are
-distributed under liberal freedom-respecting%
+\CmdStan, Stan, and Stan's four dependent libraries, Stan Math
+Library, Boost, Eigen, Sundials, and Intel TBB are distributed under
+liberal freedom-respecting%
 %
 \footnote{The link
   \url{http://www.gnu.org/philosophy/open-source-misses-the-point.html}
@@ -16,6 +17,14 @@ \chapter{Licensing}\label{licensing.appendix}
 have no ``copyleft'' provisions requiring applications of \CmdStan to
 be open source if they are redistributed.
 
+However, the Apache 2.0 license of the Intel TBB is not compatible
+with the GPL-2 (while it is compatible with GPL-3). This
+in-compatibility implies that no unitary binary of Apache 2.0 work
+(part of \CmdStan) and GPL-2 work can be distributed. A detailed
+discussion on the Apache 2.0 license implications for Stan can be
+found on our wiki
+\url{https://github.com/stan-dev/math/wiki/Apache-2.0-License-Evaluation}.
+
 This chapter describes the licenses for the tools that are distributed
 with \CmdStan.  The next chapter explains some of the build tools that
 are not distributed with \CmdStan, but are required to build and run
@@ -63,14 +72,22 @@ \section{Eigen License}
 \url{http:/http://opensource.org/licenses/mpl-2.0}
 \end{quote}
 
-\section{CVODES License} 
+\section{Sundials License} 
 %
-CVODES is distributed under the BSD 3-clause license (BSD New).
+Sundials is distributed under the BSD 3-clause license (BSD New).
 %
 \begin{quote}
 \url{http://computation.llnl.gov/projects/sundials-suite-nonlinear-differential-algebraic-equation-solvers/license}
 \end{quote}
 
+\section{Intel Threading Building Blocks License} 
+%
+The Intel TBB is distributed under the Apache 2.0 license.
+%
+\begin{quote}
+\url{https://github.com/intel/tbb/blob/tbb_2019/LICENSE}
+\end{quote}
+
 \section{Google Test License}
 
 \CmdStan uses Google Test for unit testing; it is not required to