Skip to content
This repository has been archived by the owner on Apr 17, 2023. It is now read-only.

Commit

Permalink
Merge pull request #16 from appelmar/dev
Browse files Browse the repository at this point in the history
Merge dev into master
  • Loading branch information
appelmar authored Mar 1, 2019
2 parents 8bd6d17 + abdcd39 commit 5539444
Show file tree
Hide file tree
Showing 9 changed files with 58 additions and 30 deletions.
6 changes: 3 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.8)
project(gdalcubes LANGUAGES CXX C VERSION 0.0.1)
project(gdalcubes LANGUAGES CXX C VERSION 0.1.0)

#set(COLLECTION_FORMAT_VERSION_MAJOR 0)
#set(COLLECTION_FORMAT_VERSION_MINOR 0)
Expand All @@ -19,8 +19,8 @@ endif ()
# #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg --coverage")
# #set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pg --coverage" )
# #set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pg --coverage")
# #set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=undefined -D_FORTIFY_SOURCE=0")
# #set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined -D_FORTIFY_SOURCE=0")
# set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=undefined -D_FORTIFY_SOURCE=0")
# set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined -D_FORTIFY_SOURCE=0")
#endif ( )


Expand Down
7 changes: 0 additions & 7 deletions TODO.md

This file was deleted.

8 changes: 4 additions & 4 deletions src/build_info.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
#define BUILD_INFO_H

#define GDALCUBES_VERSION_MAJOR 0
#define GDALCUBES_VERSION_MINOR 0
#define GDALCUBES_VERSION_PATCH 1
#define GDALCUBES_GIT_DESC "55146dc8"
#define GDALCUBES_GIT_COMMIT "55146dc8f0803822f3516cb9c5d7329280df1684"
#define GDALCUBES_VERSION_MINOR 1
#define GDALCUBES_VERSION_PATCH 0
#define GDALCUBES_GIT_DESC "dc8bb05a"
#define GDALCUBES_GIT_COMMIT "dc8bb05a809cc9349f282165f745da38bfc2f144"

#define COLLECTION_FORMAT_VERSION_MAJOR
#define COLLECTION_FORMAT_VERSION_MINOR
Expand Down
12 changes: 12 additions & 0 deletions src/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,18 @@ class config {
_collection_format_preset_dirs.push_back(dir);
}

std::string gdal_version_info() {
return GDALVersionInfo("--version");
}

std::vector<std::string> gdal_formats() {
std::vector<std::string> out;
for (int i = 0; i < GDALGetDriverCount(); ++i) {
out.push_back(GDALGetDriverShortName(GDALGetDriver(i)));
}
return out;
}

private:
std::shared_ptr<chunk_processor> _chunk_processor;
std::shared_ptr<progress> _progress_bar;
Expand Down
19 changes: 16 additions & 3 deletions src/cube.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ void cube::write_gtiff_directory(std::string dir, std::shared_ptr<chunk_processo
// prg->finalize();
//}

void cube::write_netcdf_file(std::string path, std::shared_ptr<chunk_processor> p) {
void cube::write_netcdf_file(std::string path, uint8_t compression_level, std::shared_ptr<chunk_processor> p) {
std::string op = filesystem::make_absolute(path);

if (filesystem::is_directory(op)) {
Expand Down Expand Up @@ -371,6 +371,11 @@ void cube::write_netcdf_file(std::string path, std::shared_ptr<chunk_processor>
for (uint16_t i = 0; i < bands().count(); ++i) {
int v;
nc_def_var(ncout, bands().get(i).name.c_str(), NC_DOUBLE, 3, d_all, &v);
std::size_t csize[3] = {_chunk_size[0], _chunk_size[1], _chunk_size[2]};
nc_def_var_chunking(ncout, v, NC_CHUNKED, csize);
if (compression_level > 0) {
nc_def_var_deflate(ncout, v, 1, 1, compression_level); // TODO: experiment with shuffling
}

if (!bands().get(i).unit.empty())
nc_put_att_text(ncout, v, "units", strlen(bands().get(i).unit.c_str()), bands().get(i).unit.c_str());
Expand Down Expand Up @@ -441,8 +446,16 @@ void chunk_processor_multithread::apply(std::shared_ptr<cube> c,
for (uint16_t it = 0; it < _nthreads; ++it) {
workers.push_back(std::thread([this, &c, f, it, &mutex](void) {
for (uint32_t i = it; i < c->count_chunks(); i += _nthreads) {
std::shared_ptr<chunk_data> dat = c->read_chunk(i);
f(i, dat, mutex);
try {
std::shared_ptr<chunk_data> dat = c->read_chunk(i);
f(i, dat, mutex);
} catch (std::string s) {
GCBS_ERROR(s);
continue;
} catch (...) {
GCBS_ERROR("unexpected exception while processing chunk " + std::to_string(i));
continue;
}
}
}));
}
Expand Down
3 changes: 2 additions & 1 deletion src/cube.h
Original file line number Diff line number Diff line change
Expand Up @@ -646,9 +646,10 @@ class cube : public std::enable_shared_from_this<cube> {
/**
* Export a cube to a single NetCDF file
* @param path path of the target file
* @param compression_level deflate level, 0=no compression, 1= fast, 9 = small
* @param p chunk processor instance, defaults to the global configuration
*/
void write_netcdf_file(std::string path, std::shared_ptr<chunk_processor> p = config::instance()->get_default_chunk_processor());
void write_netcdf_file(std::string path, uint8_t compression_level = 0, std::shared_ptr<chunk_processor> p = config::instance()->get_default_chunk_processor());

/**
* Get the cube's bands
Expand Down
28 changes: 16 additions & 12 deletions src/stream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,18 +134,6 @@ std::shared_ptr<chunk_data> stream_cube::stream_chunk_file(std::shared_ptr<chunk
std::string f_in = filesystem::join(config::instance()->get_streaming_dir(), utils::generate_unique_filename(12, ".stream_", "_in"));
std::string f_out = filesystem::join(config::instance()->get_streaming_dir(), utils::generate_unique_filename(12, ".stream_", "_out"));

#ifdef _WIN32
_putenv("GDALCUBES_STREAMING=1");
//_putenv((std::string("GDALCUBES_STREAMING_DIR") + "=" + config::instance()->get_streaming_dir().c_str()).c_str());
_putenv((std::string("GDALCUBES_STREAMING_FILE_IN") + "=" + f_in.c_str()).c_str());
_putenv((std::string("GDALCUBES_STREAMING_FILE_OUT") + "=" + f_out.c_str()).c_str());
#else
setenv("GDALCUBES_STREAMING", "1", 1);
// setenv("GDALCUBES_STREAMING_DIR", config::instance()->get_streaming_dir().c_str(), 1);
setenv("GDALCUBES_STREAMING_FILE_IN", f_in.c_str(), 1);
setenv("GDALCUBES_STREAMING_FILE_OUT", f_out.c_str(), 1);
#endif

std::string errstr; // capture error string

// write input data
Expand Down Expand Up @@ -181,10 +169,26 @@ std::shared_ptr<chunk_data> stream_cube::stream_chunk_file(std::shared_ptr<chunk
f_in_stream.write(((char *)(data->buf())), sizeof(double) * data->size()[0] * data->size()[1] * data->size()[2] * data->size()[3]);
f_in_stream.close();

/* setenv / _putenv is not thread-safe, we need to get a mutex until the child process has been started. */
static std::mutex mtx;
mtx.lock();
#ifdef _WIN32
_putenv("GDALCUBES_STREAMING=1");
//_putenv((std::string("GDALCUBES_STREAMING_DIR") + "=" + config::instance()->get_streaming_dir().c_str()).c_str());
_putenv((std::string("GDALCUBES_STREAMING_FILE_IN") + "=" + f_in.c_str()).c_str());
_putenv((std::string("GDALCUBES_STREAMING_FILE_OUT") + "=" + f_out.c_str()).c_str());
#else
setenv("GDALCUBES_STREAMING", "1", 1);
// setenv("GDALCUBES_STREAMING_DIR", config::instance()->get_streaming_dir().c_str(), 1);
setenv("GDALCUBES_STREAMING_FILE_IN", f_in.c_str(), 1);
setenv("GDALCUBES_STREAMING_FILE_OUT", f_out.c_str(), 1);
#endif

// start process
TinyProcessLib::Process process(_cmd, "", [](const char *bytes, std::size_t n) {}, [&errstr](const char *bytes, std::size_t n) {
errstr = std::string(bytes, n);
GCBS_DEBUG(errstr); }, false);
mtx.unlock();
auto exit_status = process.get_exit_status();
filesystem::remove(f_in);
if (exit_status != 0) {
Expand Down
4 changes: 4 additions & 0 deletions src/utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <gdal_priv.h>
#include <iomanip>
#include <iostream>
#include <mutex>
#include <random>
#include <sstream>
#include <string>
Expand All @@ -40,11 +41,14 @@ class utils {
static std::mt19937 gen(time(NULL)); //Standard mersenne_twister_engine seeded with rd()
static const std::string LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
static std::uniform_int_distribution<> dis(0, LETTERS.length() - 1);
static std::mutex mtx;
mtx.lock();
std::stringstream ss;
for (uint16_t i = 0; i < n; ++i) {
ss << LETTERS[dis(gen)];
}
std::string out = prefix + ss.str() + suffix;
mtx.unlock();
return out;
}

Expand Down
1 change: 1 addition & 0 deletions src/window_time.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ std::shared_ptr<chunk_data> window_time_cube::read_chunk(chunkid_t id) {
// buffer for a single time series including data from adjacent chunks for all used input bands
uint32_t cur_ts_length = _win_size_l + size_tyx[0] + _win_size_r;
double* cur_ts = (double*)std::calloc(cur_ts_length * _bands.count(), sizeof(double));
std::fill(cur_ts, cur_ts + cur_ts_length * _bands.count(), NAN);

for (uint32_t ixy = 0; ixy < size_tyx[1] * size_tyx[2]; ++ixy) {
// fill values from l chunks
Expand Down

0 comments on commit 5539444

Please sign in to comment.