implement BridgeStan download and module compilation on Rust

.github/workflows/main.yaml

@@ -3,7 +3,7 @@ name: bridgestan tests
 on:
  push:
  branches:
- - 'main'
+ - "main"
  pull_request:
  workflow_dispatch: {}
 
@@ -128,7 +128,6 @@ jobs:
  env:
  BRIDGESTAN: ${{ github.workspace }}
 
-
  julia:
  needs: [build]
  runs-on: ${{matrix.os}}
@@ -270,12 +269,6 @@ jobs:
  path: ./test_models/
  key: ${{ hashFiles('**/*.stan', 'src/*', 'stan/src/stan/version.hpp', 'Makefile') }}-${{ matrix.os }}-v${{ env.CACHE_VERSION }}
 
- - name: Install LLVM and Clang
- uses: KyleMayes/install-llvm-action@v2
- with:
- version: "15.0"
- directory: ${{ runner.temp }}/llvm
-
  - name: Set up TBB
  if: matrix.os == 'windows-latest'
  run: |
@@ -284,11 +277,11 @@ jobs:
  - name: Run rust tests
  working-directory: ./rust
  timeout-minutes: 60
- env:
- LIBCLANG_PATH: ${{ runner.temp }}/llvm/lib
- LLVM_CONFIG_PATH: ${{ runner.temp }}/llvm/bin/llvm-config
  run: |
  cargo clippy
  cargo fmt --check
  cargo run --example=example
- cargo test --verbose
+
+ # run all tests with feature download-bridgestan-src
+ cargo test --verbose --all-features
+ cargo test --verbose model_compiling -- --ignored

.gitignore

@@ -12,6 +12,7 @@ c-example/example_static
 # Rust
 rust/target/
 rust/Cargo.lock
+rust/.vscode
 
 notes.org
 

docs/languages/rust.rst

@@ -14,13 +14,9 @@ The BridgeStan Rust client is available on `crates.io <https://crates.io/crates/
 
  cargo add bridgestan
 
-To build and use BridgeStan models, a copy of the BridgeStan C++ source code
-is required. Please follow the :doc:`Getting Started guide <../getting-started>`
-or use the Rust client in tandem with an interface such as :doc:`Python <./python>`
-which automates this process.
+The first time you compile a model, the BridgeStan source code will be downloaded to `~/.bridgestan`. If you prefer to use a source distribution of BridgeStan, you can pass its path as the `bs_path` argument to `compile_model`.
 
-``STAN_THREADS=true`` needs to be specified when compiling a model, for more
-details see the `API reference <https://docs.rs/bridgestan>`__.
+Note that the system pre-requisites from the [Getting Started Guide](../getting-started.rst) are still required and will not be automatically installed by this method.
 
 Example Program
 ---------------

rust/Cargo.toml

@@ -11,10 +11,23 @@ homepage = "https://roualdes.github.io/bridgestan/latest/"
 [dependencies]
 libloading = "0.8.0"
 thiserror = "1.0.40"
+path-absolutize = { version = "3.1" }
+log = { version = "0.4" }
+ureq = { version = "2.7", optional = true }
+tar = { version = "0.4", optional = true }
+flate2 = { version = "1.0", optional = true }
+dirs = { version = "5.0", optional = true }
+
+[features]
+download-bridgestan-src = ["dep:ureq", "dep:tar", "dep:flate2", "dep:dirs"]
 
 [build-dependencies]
 bindgen = "0.69.1"
 
 [dev-dependencies]
 approx = "0.5.1"
 rand = "0.8.5"
+env_logger = "0.11"
+
+[[example]]
+name = "example"

rust/README.md

@@ -11,42 +11,40 @@ Internally, it relies on [`bindgen`](https://docs.rs/bindgen/) and
 
 ## Compiling the model
 
-The Rust wrapper does not currently have any functionality to compile Stan models.
-Compiled shared libraries need to be built manually using `make` or with the Julia
-or Python bindings.
+The Rust wrapper has the ability to compile Stan models by invoking the `make` command through the [`compile_model`] function.
 
-For safety reasons all Stan models need to be installed with `STAN_THREADS=true`.
-When compiling a model using `make`, set the environment variable:
+This requires a C++ toolchain and a copy of the BridgeStan source code. The source code can be downloaded automatically by enabling the `download-bridgestan-src` feature and calling [`download_bridgestan_src`]. Alternatively, the path to the BridgeStan source code can be provided manually.
 
-```bash
-STAN_THREADS=true make some_model
-```
-
-When compiling a Stan model in python, this has to be specified in the `make_args`
-argument:
-
-```python
-path = bridgestan.compile_model("stan_model.stan", make_args=["STAN_THREADS=true"])
-```
+For safety reasons all Stan models need to be built with `STAN_THREADS=true`. This is the default behavior in the `compile_model` function,
+but may need to be set manually when compiling the model in other contexts.
 
 If `STAN_THREADS` was not specified while building the model, the Rust wrapper
 will throw an error when loading the model.
 
-## Usage:
+## Usage
 
 Run this example with `cargo run --example=example`.
 
 ```rust
 use std::ffi::CString;
-use std::path::Path;
-use bridgestan::{BridgeStanError, Model, open_library};
+use std::path::{Path, PathBuf};
+use bridgestan::{BridgeStanError, Model, open_library, compile_model};
 
-// The path to the compiled model.
-// Get for instance from python `bridgestan.compile_model`
+// The path to the Stan model
 let path = Path::new(env!["CARGO_MANIFEST_DIR"])
  .parent()
  .unwrap()
- .join("test_models/simple/simple_model.so");
+ .join("test_models/simple/simple.stan");
+
+// You can manually set the BridgeStan src path or
+// automatically download it (but remember to
+// enable the download-bridgestan-src feature first)
+let bs_path: PathBuf = "..".into();
+// let bs_path = bridgestan::download_bridgestan_src().unwrap();
+
+// The path to the compiled model
+let path = compile_model(&bs_path, &path, &[], &[]).expect("Could not compile Stan model.");
+println!("Compiled model: {:?}", path);
 
 let lib = open_library(path).expect("Could not load compiled Stan model.");
 
@@ -59,11 +57,13 @@ let data = CString::new(data.to_string().into_bytes()).unwrap();
 let seed = 42;
 
 let model = match Model::new(&lib, Some(data), seed) {
-Ok(model) => { model },
-Err(BridgeStanError::ConstructFailed(msg)) => {
- panic!("Model initialization failed. Error message from Stan was {}", msg)
-},
-_ => { panic!("Unexpected error") },
+ Ok(model) => model,
+ Err(BridgeStanError::ConstructFailed(msg)) => {
+ panic!("Model initialization failed. Error message from Stan was {msg}")
+ }
+ Err(e) => {
+ panic!("Unexpected error:\n{e}")
+ }
 };
 
 let n_dim = model.param_unc_num();

rust/examples/example.rs

@@ -1,14 +1,31 @@
-use bridgestan::{open_library, BridgeStanError, Model};
+use bridgestan::{compile_model, open_library, BridgeStanError, Model};
 use std::ffi::CString;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 fn main() {
- // The path to the compiled model.
- // Get for instance from python `bridgestan.compile_model`
+ // Set up logging - optional
+ if std::env::var("RUST_LOG").is_err() {
+ std::env::set_var("RUST_LOG", "bridgestan=info");
+ }
+ env_logger::init();
+
+ // The path to the Stan model
  let path = Path::new(env!["CARGO_MANIFEST_DIR"])
  .parent()
  .unwrap()
- .join("test_models/simple/simple_model.so");
+ .join("test_models")
+ .join("simple")
+ .join("simple.stan");
+
+ // You can manually set the BridgeStan src path or
+ // automatically download it (but remember to
+ // enable the download-bridgestan-src feature first)
+ let bs_path: PathBuf = "..".into();
+ // let bs_path = bridgestan::download_bridgestan_src().unwrap();
+
+ // The path to the compiled model
+ let path = compile_model(&bs_path, &path, &[], &[]).expect("Could not compile Stan model.");
+ println!("Compiled model: {:?}", path);
 
  let lib = open_library(path).expect("Could not load compiled Stan model.");
 

rust/src/bs_safe.rs

@@ -1,4 +1,5 @@
 use crate::ffi;
+use crate::VERSION;
 use std::borrow::Borrow;
 use std::collections::hash_map::DefaultHasher;
 use std::ffi::c_char;
@@ -101,9 +102,15 @@ pub enum BridgeStanError {
  /// Setting a print-callback failed.
  #[error("Failed to set a print-callback: {0}")]
  SetCallbackFailed(String),
+ /// Compilation of the Stan model shared object failed.
+ #[error("Failed to compile Stan model: {0}")]
+ ModelCompilingFailed(String),
+ /// Downloading BridgeStan's C++ source code from GitHub failed.
+ #[error("Failed to download BridgeStan {VERSION} from github.com: {0}")]
+ DownloadFailed(String),
 }
 
-type Result<T> = std::result::Result<T, BridgeStanError>;
+pub(crate) type Result<T> = std::result::Result<T, BridgeStanError>;
 
 /// Open a compiled Stan library.
 ///

rust/src/compile.rs

@@ -0,0 +1,91 @@
+use crate::bs_safe::{BridgeStanError, Result};
+use log::info;
+use path_absolutize::Absolutize;
+use std::path::{Path, PathBuf};
+
+const MAKE: &str = if cfg!(target_os = "windows") {
+ "mingw32-make"
+} else {
+ "make"
+};
+
+/// Compile a Stan Model. Requires a path to the BridgeStan sources (can be
+/// downloaded with [`download_bridgestan_src`](crate::download_bridgestan_src) if that feature
+/// is enabled), a path to the `.stan` file, and additional arguments
+/// for the Stan compiler and the make command.
+pub fn compile_model(
+ bs_path: &Path,
+ stan_file: &Path,
+ stanc_args: &[&str],
+ make_args: &[&str],
+) -> Result<PathBuf> {
+ // using path_absolutize crate for now since
+ // std::fs::canonicalize doesn't behave well on windows
+ // we may switch to std::path::absolute once it stabilizes, see
+ // https://github.com/roualdes/bridgestan/pull/212#discussion_r1513375667
+ let stan_file = stan_file
+ .absolutize()
+ .map_err(|e| BridgeStanError::ModelCompilingFailed(e.to_string()))?;
+
+ // get --include-paths=model_dir
+ let includir_stan_file_dir = stan_file
+ .parent()
+ .and_then(Path::to_str)
+ .map(|x| format!("--include-paths={x}"))
+ .unwrap_or_default();
+
+ let includir_stan_file_dir = includir_stan_file_dir.as_str();
+
+ if stan_file.extension().unwrap_or_default() != "stan" {
+ return Err(BridgeStanError::ModelCompilingFailed(
+ "File must be a .stan file".to_owned(),
+ ));
+ }
+
+ // add _model suffix and change extension to .so
+ let output = stan_file.with_extension("");
+ let output = output.with_file_name(format!(
+ "{}_model",
+ output.file_name().unwrap_or_default().to_string_lossy()
+ ));
+ let output = output.with_extension("so");
+
+ let stanc_args = [&[includir_stan_file_dir], stanc_args].concat();
+ let stanc_args = stanc_args.join(" ");
+ let stanc_args = format!("STANCFLAGS={}", stanc_args);
+ let stanc_args = [stanc_args.as_str()];
+
+ let cmd = [
+ &[output.to_str().unwrap_or_default()],
+ make_args,
+ stanc_args.as_slice(),
+ ]
+ .concat();
+
+ info!(
+ "Compiling model with command: {} \"{}\"",
+ MAKE,
+ cmd.join("\" \"")
+ );
+ std::process::Command::new(MAKE)
+ .args(cmd)
+ .current_dir(bs_path)
+ .env("STAN_THREADS", "true")
+ .output()
+ .map_err(|e| e.to_string())
+ .and_then(|proc| {
+ if !proc.status.success() {
+ Err(format!(
+ "{} {}",
+ String::from_utf8_lossy(proc.stdout.as_slice()).into_owned(),
+ String::from_utf8_lossy(proc.stderr.as_slice()).into_owned(),
+ ))
+ } else {
+ Ok(())
+ }
+ })
+ .map_err(|e| BridgeStanError::ModelCompilingFailed(e.to_string()))?;
+ info!("Finished compiling model");
+
+ Ok(output)
+}

rust/src/download.rs

@@ -0,0 +1,62 @@
+use crate::bs_safe::{BridgeStanError, Result};
+use crate::VERSION;
+use flate2::read::GzDecoder;
+use log::info;
+use std::{env::temp_dir, fs, path::PathBuf};
+use tar::Archive;
+
+/// Download and unzip the BridgeStan source distribution for this version
+/// to `~/.bridgestan/bridgestan-$VERSION`.
+pub fn download_bridgestan_src() -> Result<PathBuf> {
+ let homedir = dirs::home_dir().unwrap_or(temp_dir());
+
+ let bs_path_download_temp = homedir.join(".bridgestan_tmp_dir");
+ let bs_path_download = homedir.join(".bridgestan");
+
+ let bs_path_download_temp_join_version =
+ bs_path_download_temp.join(format!("bridgestan-{VERSION}"));
+ let bs_path_download_join_version = bs_path_download.join(format!("bridgestan-{VERSION}"));
+
+ if !bs_path_download_join_version.exists() {
+ info!("Downloading BridgeStan");
+
+ fs::remove_dir_all(&bs_path_download_temp).unwrap_or_default();
+ fs::create_dir(&bs_path_download_temp).unwrap_or_default();
+ fs::create_dir(&bs_path_download).unwrap_or_default();
+
+ let url = "https://github.com/roualdes/bridgestan/releases/download/".to_owned()
+ + format!("v{VERSION}/bridgestan-{VERSION}.tar.gz").as_str();
+
+ let response = ureq::get(url.as_str())
+ .call()
+ .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+ let len = response
+ .header("Content-Length")
+ .and_then(|s| s.parse::<usize>().ok())
+ .unwrap_or(50_000_000);
+
+ let mut bytes: Vec<u8> = Vec::with_capacity(len);
+ response
+ .into_reader()
+ .read_to_end(&mut bytes)
+ .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+ let tar = GzDecoder::new(bytes.as_slice());
+ let mut archive = Archive::new(tar);
+ archive
+ .unpack(&bs_path_download_temp)
+ .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+ fs::rename(
+ bs_path_download_temp_join_version,
+ &bs_path_download_join_version,
+ )
+ .map_err(|e| BridgeStanError::DownloadFailed(e.to_string()))?;
+
+ fs::remove_dir(bs_path_download_temp).unwrap_or_default();
+
+ info!("Finished downloading BridgeStan");
+ }
+
+ Ok(bs_path_download_join_version)
+}