Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert byte_no_cf and cf_nasa_4326 to Zarr #321

Merged
merged 3 commits into from
Oct 21, 2022
Merged

Convert byte_no_cf and cf_nasa_4326 to Zarr #321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e6a6b4e
Convert byte_no_cf and cf_nasa_4326 to Zarr
lnicola Oct 19, 2022
d9b6dea
Ignore Zarr tests on pre-GDAL 3.4
lnicola Oct 20, 2022
a61bf52
Delete nc fixtures
lnicola Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file removed fixtures/byte_no_cf.nc
View file
Open in desktop
Binary file not shown.
Binary file added fixtures/byte_no_cf.zarr.zip
View file
Open in desktop
Binary file not shown.
Binary file removed fixtures/cf_nasa_4326.nc
View file
Open in desktop
Binary file not shown.
Binary file added fixtures/cf_nasa_4326.zarr.zip
View file
Open in desktop
Binary file not shown.
9 changes: 5 additions & 4 deletions src/programs/raster/mdimtranslate.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,11 +230,12 @@ mod tests {

use super::*;

use crate::{test_utils::TempFixture, DatasetOptions, Driver, GdalOpenFlags};
use crate::{DatasetOptions, Driver, GdalOpenFlags};

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_build_tiff_from_path() {
let fixture = TempFixture::fixture("cf_nasa_4326.nc");
let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -243,7 +244,6 @@ mod tests {
sibling_files: None,
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();

let mem_file_path = "/vsimem/2d3e9124-a7a0-413e-97b5-e79d46e50ff8";

let dataset = multi_dim_translate(
Expand All @@ -265,8 +265,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_build_tiff_from_dataset() {
let fixture = TempFixture::fixture("cf_nasa_4326.nc");
let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand Down
93 changes: 57 additions & 36 deletions src/raster/mdarray.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,8 +803,9 @@ mod tests {
use crate::{test_utils::TempFixture, Dataset, DatasetOptions, GdalOpenFlags};

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_root_group_name() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -819,8 +820,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_array_names() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -830,14 +832,18 @@ mod tests {
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
let root_group = dataset.root_group().unwrap();
let options = CslStringList::new(); //Driver specific options determining how groups should be retrieved. Pass nullptr for default behavior.
let options = CslStringList::new();
let array_names = root_group.array_names(options);
assert_eq!(array_names, vec!["Band1".to_string()])
assert_eq!(
array_names,
vec!["X".to_string(), "Y".to_string(), "byte_no_cf".to_string()]
)
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_n_dimension() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -847,16 +853,17 @@ mod tests {
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
let root_group = dataset.root_group().unwrap();
let array_name = "Band1".to_string();
let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
let array_name = "byte_no_cf".to_string();
let options = CslStringList::new();
let md_array = root_group.open_md_array(&array_name, options).unwrap();
let n_dimension = md_array.num_dimensions();
assert_eq!(2, n_dimension);
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_n_elements() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -866,16 +873,17 @@ mod tests {
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
let root_group = dataset.root_group().unwrap();
let array_name = "Band1".to_string();
let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
let array_name = "byte_no_cf".to_string();
let options = CslStringList::new();
let md_array = root_group.open_md_array(&array_name, options).unwrap();
let n_elements = md_array.num_elements();
assert_eq!(400, n_elements);
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_dimension_name() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -892,24 +900,25 @@ mod tests {
.into_iter()
.map(|dimensions| dimensions.name())
.collect();
assert_eq!(group_dimensions_names, vec!["x", "y"]);
assert_eq!(group_dimensions_names, vec!["X", "Y"]);

// array dimensions

let array_name = "Band1".to_string();
let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
let array_name = "byte_no_cf".to_string();
let options = CslStringList::new();
let md_array = root_group.open_md_array(&array_name, options).unwrap();
let dimensions = md_array.dimensions().unwrap();
let mut dimension_names = Vec::new();
for dimension in dimensions {
dimension_names.push(dimension.name());
}
assert_eq!(dimension_names, vec!["y".to_string(), "x".to_string()])
assert_eq!(dimension_names, vec!["Y".to_string(), "X".to_string()])
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_dimension_size() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -919,8 +928,8 @@ mod tests {
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
let root_group = dataset.root_group().unwrap();
let array_name = "Band1".to_string();
let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
let array_name = "byte_no_cf".to_string();
let options = CslStringList::new();
let md_array = root_group.open_md_array(&array_name, options).unwrap();
let dimensions = md_array.dimensions().unwrap();
let mut dimensions_size = Vec::new();
Expand All @@ -931,8 +940,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_read_data() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -944,7 +954,7 @@ mod tests {

let root_group = dataset.root_group().unwrap();
let md_array = root_group
.open_md_array("Band1", CslStringList::new())
.open_md_array("byte_no_cf", CslStringList::new())
.unwrap();

let values = md_array.read_as::<u8>(vec![0, 0], vec![20, 20]).unwrap();
Expand All @@ -956,7 +966,10 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_read_string_array() {
// Beware https://github.com/georust/gdal/issues/299 if you want to reuse this
// This can't be Zarr because it doesn't support string arrays
let fixture = TempFixture::fixture("alldatatypes.nc");

let dataset_options = DatasetOptions {
Expand Down Expand Up @@ -986,8 +999,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_datatype() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -1000,7 +1014,7 @@ mod tests {
let root_group = dataset.root_group().unwrap();

let md_array = root_group
.open_md_array("Band1", CslStringList::new())
.open_md_array("byte_no_cf", CslStringList::new())
.unwrap();

let datatype = md_array.datatype();
Expand All @@ -1011,8 +1025,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_spatial_ref() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -1024,7 +1039,7 @@ mod tests {

let root_group = dataset.root_group().unwrap();
let md_array = root_group
.open_md_array("Band1", CslStringList::new())
.open_md_array("byte_no_cf", CslStringList::new())
.unwrap();

let spatial_ref = md_array.spatial_reference().unwrap();
Expand All @@ -1035,8 +1050,9 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_no_data_value() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
let fixture = "/vsizip/fixtures/byte_no_cf.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand All @@ -1048,15 +1064,16 @@ mod tests {

let root_group = dataset.root_group().unwrap();
let md_array = root_group
.open_md_array("Band1", CslStringList::new())
.open_md_array("byte_no_cf", CslStringList::new())
.unwrap();

assert_eq!(md_array.no_data_value_as_double(), Some(0.));
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_attributes() {
let fixture = TempFixture::fixture("cf_nasa_4326.nc");
let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand Down Expand Up @@ -1101,15 +1118,13 @@ mod tests {
"air_temperature"
);

assert_eq!(
md_array.attribute("_FillValue").unwrap().read_as_f64(),
-9999.
);
assert_eq!(md_array.no_data_value_as_double().unwrap(), -9999.);
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_unit() {
let fixture = TempFixture::fixture("cf_nasa_4326.nc");
let fixture = "/vsizip/fixtures/cf_nasa_4326.zarr.zip";

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
Expand Down Expand Up @@ -1147,19 +1162,25 @@ mod tests {
}

#[test]
#[cfg_attr(not(all(major_ge_3, minor_ge_4)), ignore)]
fn test_stats() {
let fixture = TempFixture::fixture("byte_no_cf.nc");
// make a copy to avoid writing the statistics into the original file
let fixture = TempFixture::fixture("byte_no_cf.zarr.zip");

let dataset_options = DatasetOptions {
open_flags: GdalOpenFlags::GDAL_OF_MULTIDIM_RASTER,
allowed_drivers: None,
open_options: None,
sibling_files: None,
};
let dataset = Dataset::open_ex(&fixture, dataset_options).unwrap();
let dataset = Dataset::open_ex(
format!("/vsizip/{}", fixture.path().display()),
dataset_options,
)
.unwrap();
let root_group = dataset.root_group().unwrap();
let array_name = "Band1".to_string();
let options = CslStringList::new(); //Driver specific options determining how the array should be opened. Pass nullptr for default behavior.
let array_name = "byte_no_cf".to_string();
let options = CslStringList::new();
let md_array = root_group.open_md_array(&array_name, options).unwrap();

assert!(md_array.get_statistics(false, true).unwrap().is_none());
Expand Down