Merge pull request #56 from magnusuMET/vendor

Option for building netCDF from source

.github/workflows/ci.yml

@@ -18,6 +18,9 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with: {submodules: true}
+    - name: Install netCDF
+      run: sudo apt-get install libnetcdf-dev
     - name: Install rust
       uses: actions-rs/toolchain@v1
       with:
@@ -28,13 +31,13 @@ jobs:
     - name: Check formatting
       run: cargo fmt -- --check
     - name: Documentation
-      run: cargo doc --workspace
+      run: cargo doc --workspace --exclude netcdf-src
     - name: Clippy
-      run: cargo clippy --workspace -- -D warnings
+      run: cargo clippy --workspace --exclude netcdf-src -- -D warnings
 
-  test:
-    name: test
-    runs-on: ${{ matrix.os }}
+  test_apt:
+    name: test apt
+    runs-on: ubuntu-18.04
     strategy:
       matrix:
         build:
@@ -43,17 +46,15 @@ jobs:
         - nightly
         include:
         - build: stable
-          os: ubuntu-18.04
           rust: stable
         - build: beta
-          os: ubuntu-18.04
           rust: beta
         - build: nightly
-          os: ubuntu-18.04
           rust: nightly
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with: {submodules: false}
 
     - name: Install netcdf
       run: sudo apt-get install libnetcdf-dev
@@ -66,37 +67,63 @@ jobs:
         override: true
 
     - name: Build
-      run: cargo build --verbose --workspace
+      run: cargo build --verbose --workspace --exclude netcdf-src
 
     - name: Test
-      run: cargo test --verbose --workspace
+      run: cargo test --verbose --workspace --exclude netcdf-src
       if: matrix.os == 'ubuntu-18.04'
 
-  tarpaulin:
-    name: tarpaulin
-    runs-on: ubuntu-18.04
+  conda:
+    name: conda
+    runs-on: ${{matrix.os}}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu, channel: conda-forge, rust: stable}
+          - {os: windows, channel: conda-forge, rust: stable}
+          - {os: macos, channel: conda-forge, rust: stable}
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-    - name: Install netcdf
-      run: sudo apt-get install libnetcdf-dev
-    - name: Install rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        profile: minimal
-        override: true
-    - name: Install tarpaulin
-      uses: actions-rs/install@v0.1
-      with:
-        crate: cargo-tarpaulin
-        version: latest
-        use-tool-cache: true
-
-    - name: Tarpaulin
-      run: cargo tarpaulin --verbose --out Xml --ignore-tests
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with: {submodules: true}
+      - name: Install Rust (${{matrix.rust}})
+        uses: actions-rs/toolchain@v1
+        with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true}
+      - name: Install conda
+        uses: goanpeca/setup-miniconda@v1
+        with: {auto-update-conda: false, activate-environment: testenv}
+      - name: Install netCDF
+        run: conda install -y -c ${{matrix.channel}} libnetcdf=4.7.4
+      - name: Build and test
+        run: |
+          export HDF5_DIR="$CONDA_PREFIX"
+          [ "${{runner.os}}" != "Windows" ] && export RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib"
+          cargo test -vv --workspace --exclude netcdf-src
 
-    - name: Upload to codecov
-      uses: codecov/codecov-action@v1.0.2
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
+  static_builds:
+    name: static builds
+    runs-on: ${{matrix.os}}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {os: ubuntu, rust: stable}
+          - {os: windows, rust: stable-msvc}
+          - {os: windows, rust: stable-gnu}
+          - {os: macos, rust: stable}
+    defaults:
+      run:
+          shell: bash -l {0}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with: {submodules: true}
+      - name: Install Rust (${{matrix.rust}})
+        uses: actions-rs/toolchain@v1
+        with: {toolchain: '${{matrix.rust}}', profile: minimal, override: true}
+      - name: Build and test
+        run: cargo test -vv --workspace --features static

.github/workflows/codecov.yml

@@ -0,0 +1,34 @@
+name: codecov
+on: [push]
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  tarpaulin:
+    name: tarpaulin
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Install netcdf
+      run: sudo apt-get install libnetcdf-dev
+    - name: Install rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        profile: minimal
+        override: true
+    - name: Install tarpaulin
+      uses: actions-rs/install@v0.1
+      with:
+        crate: cargo-tarpaulin
+        version: latest
+        use-tool-cache: true
+
+    - name: Tarpaulin
+      run: cargo tarpaulin --verbose --out Xml --ignore-tests
+
+    - name: Upload to codecov
+      uses: codecov/codecov-action@v1.0.2
+      with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "netcdf-sys/source"]
+	path = netcdf-src/source
+	url = https://github.com/Unidata/netcdf-c.git

Cargo.toml

@@ -17,6 +17,7 @@ exclude = ["examples/**", "tests/**"]
 [features]
 default = ["ndarray"]
 memory = ["netcdf-sys/memio"]
+static = ["netcdf-sys/static"]
 
 [dependencies]
 lazy_static = "1.4.0"
@@ -30,4 +31,5 @@ structopt = "0.3.3"
 [workspace]
 members = [
     "netcdf-sys",
+    "netcdf-src",
 ]

README.md

@@ -9,6 +9,7 @@
 <!-- [![dependency status](https://deps.rs/repo/github/georust/netcdf/status.svg)](https://deps.rs/repo/github/georust/netcdf) -->
 
 Medium-level [netCDF](http://www.unidata.ucar.edu/software/netcdf/) bindings for Rust, allowing easy reading and writing of array-like structures to a file.
+netCDF can read and write `hdf5` files, which is a commonly used file format in scientific computing.
 
 ## Status
 
@@ -32,14 +33,10 @@ All variable data is read into a contiguous buffer, or inta an [ndarray](https:/
 
 ## Building
 
-This crate depends on libnetcdf. The Travis build runs on Ubuntu 16.04 Xenial and installs libnetcdf via apt, which results in netcdf v.4.4.0. netcdf is not widely tested on other versions of netcdf.
+This crate depends on `libnetcdf`, but a static build from source is also supported, which can be enabled using the `static` feature.
 
-You can build the library and run the tests via Docker like this:
+The crate is built on several platforms using github actions, and is currently known to build form from source on all major platforms (linux, macos, windows (gnu+msvc)), and through the package installers `conda` and `apt`.
 
-```
-docker build . -t netcdf
-docker run -it --rm netcdf
-```
 
 ## Documentation
 

netcdf-src/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "netcdf-src"
+version = "0.1.0"
+authors = ["Magnus Ulimoen <magnusu@met.no>"]
+edition = "2018"
+description = "Build scripts for building `netCDF` from source"
+build = "build.rs"
+repository = "https://github.com/georust/netcdf"
+license-file = "source/COPYRIGHT"
+links = "netcdfsrc"
+exclude = [
+    "source/unit_test/**",
+    "source/NUG/**",
+    "source/dap4_test/**",
+    "source/examples/**",
+    "source/nc_test/**",
+    "source/h5_test/**",
+    "source/nc_perf/**",
+    "source/ncdump/**",
+    "source/hdf4_test/**",
+    "source/ncgen/**",
+    "source/ncgen3/**",
+    "source/nctest/**",
+]
+
+[features]
+dap = []
+
+[dependencies]
+hdf5-sys = { version = "0.7.0", features = ["hl", "deprecated", "zlib"] }
+
+[build-dependencies]
+cmake = "0.1.44"

netcdf-src/build.rs

@@ -0,0 +1,49 @@
+macro_rules! feature {
+    ($feature:expr) => {
+        std::env::var(concat!("CARGO_FEATURE_", $feature))
+    };
+}
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+
+    let hdf5_incdir = std::env::var("DEP_HDF5_INCLUDE").unwrap();
+    let hdf5_lib = std::env::var("DEP_HDF5_LIBRARY").unwrap();
+    let hdf5_hl_lib = std::env::var("DEP_HDF5_HL_LIBRARY").unwrap();
+
+    let mut netcdf_config = cmake::Config::new("source");
+    netcdf_config
+        .define("BUILD_SHARED_LIBS", "OFF")
+        .define("NC_FIND_SHARED_LIBS", "OFF")
+        .define("BUILD_UTILITIES", "OFF")
+        .define("ENABLE_EXAMPLES", "OFF")
+        .define("ENABLE_DAP_REMOTE_TESTS", "OFF")
+        .define("ENABLE_TESTS", "OFF")
+        .define("ENABLE_EXTREME_NUMBERS", "OFF")
+        .define("ENABLE_PARALLEL_TESTS", "OFF")
+        .define("ENABLE_FILTER_TESTING", "OFF")
+        .define("ENABLE_BASH_SCRIPT_TESTING", "OFF")
+        //
+        .define("HDF5_C_LIBRARY", &hdf5_lib)
+        .define("HDF5_HL_LIBRARY", &hdf5_hl_lib)
+        .define("HDF5_INCLUDE_DIR", hdf5_incdir)
+        //
+        .define("ENABLE_DAP", "OFF") // TODO: feature flag, requires curl
+        //
+        .profile("RelWithDebInfo"); // TODO: detect opt-level
+
+    if feature!("DAP").is_ok() {
+        netcdf_config.define("ENABLE_DAP", "ON");
+    }
+
+    let netcdf = netcdf_config.build();
+
+    println!("cargo:lib={}", "netcdf");
+    let search_path = format!("{}/lib", netcdf.display());
+    if std::path::Path::new(&search_path).exists() {
+        println!("cargo:search={}", search_path);
+    } else {
+        let search_path = format!("{}/lib64", netcdf.display());
+        println!("cargo:search={}", search_path);
+    }
+}

netcdf-src/src/lib.rs

@@ -0,0 +1,3 @@
+//! Dummy crate for building `netCDF` from source
+//!
+//! The current pinned version is 4.7.4

netcdf-sys/Cargo.toml

@@ -8,15 +8,44 @@ authors = [
 license = "MIT"
 description = "FFI bindings to NetCDF"
 repository = "https://github.com/georust/netcdf"
-documentation = "https://github.com/georust/netcdf"
+documentation = "https://docs.rs/netcdf"
 keywords = ["netcdf", "hdf", "hdf5", "cdm", "ffi"]
 edition = "2018"
 links = "netcdf"
 build = "build.rs"
-exclude = ["testdata/**"]
+readme = "README.md"
+exclude = [
+    "testdata/**",
+    "source/conda.recipe/**",
+    "source/ctest_scripts/**",
+    "source/dap4_test/**",
+#   "source/docs/**",
+    "source/debug/**",
+    "source/examples/**",
+    "source/h5test/**",
+    "source/ncdump/**",
+    "source/ncgen/**",
+    "source/ncgen3/**",
+    "source/nc_perf/**",
+    "source/nctest/**",
+    "source/nc_test/**",
+#   "source/nc_test4/**",
+    "source/NUG/**",
+    "source/unit_test/**",
+]
+
+[dependencies]
+libz-sys = { version = "1.0.25" }
+curl-sys = { version = "0.4.25", optional = true }
+hdf5-sys = { version = "0.7.0" }
+netcdf-src = { path = "../netcdf-src", version = "0.1.0", optional = true }
 
 [dev-dependencies]
 lazy_static = "1.4.0"
 
 [features]
+default = []
 memio = []
+static = ["libz-sys/static", "hdf5-sys/static", "hdf5-sys/hl", "hdf5-sys/deprecated", "hdf5-sys/zlib", "netcdf-src"]
+# TODO: Link static curl (but only when using "dap") "curl-sys/static-curl", "curl-sys/static-ssl"
+dap = ["curl-sys"]

netcdf-sys/README.md

@@ -0,0 +1,4 @@
+# netcdf-sys
+
+Rust bindings to `netcdf-c` to locate and link the system libraries neccessary to use `netcdf`.
+This library can also build `hdf5` and `netcdf` from source, to allow a fully static linking experience. This is enabled with the `static` feature.