Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve I/O error handling #3321

Merged
merged 1 commit into from
Dec 1, 2016
Merged

Improve I/O error handling #3321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 5 additions & 10 deletions include/extractor/raster_source.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include <boost/filesystem/fstream.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_int.hpp>
#include <storage/io.hpp>

#include <iterator>
#include <unordered_map>
Expand Down Expand Up @@ -43,20 +44,14 @@ class RasterGrid
ydim = _ydim;
_data.reserve(ydim * xdim);

boost::filesystem::ifstream stream(filepath, std::ios::binary);
if (!stream)
{
throw util::exception("Unable to open raster file.");
}
storage::io::FileReader file_reader(filepath, storage::io::FileReader::HasNoFingerprint);

stream.seekg(0, std::ios_base::end);
std::string buffer;
buffer.resize(static_cast<std::size_t>(stream.tellg()));

stream.seekg(0, std::ios_base::beg);
buffer.resize(file_reader.Size());

BOOST_ASSERT(buffer.size() > 1);
stream.read(&buffer[0], static_cast<std::streamsize>(buffer.size()));

file_reader.ReadInto(&buffer[0], buffer.size());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of the pattern

string b;
b.resize(reader.Size())
reader.ReadInto(&b[0], b.size());

why not abstract over this to let the reader already return a type filled with the contents?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ticketing and will tackle in a different PR.


boost::algorithm::trim(buffer);

Expand Down
168 changes: 29 additions & 139 deletions include/storage/io.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,9 @@
#ifndef OSRM_STORAGE_IO_HPP_
#define OSRM_STORAGE_IO_HPP_

#include "contractor/query_edge.hpp"
#include "extractor/extractor.hpp"
#include "extractor/original_edge_data.hpp"
#include "extractor/query_node.hpp"
#include "util/exception.hpp"
#include "util/fingerprint.hpp"
#include "util/simple_logger.hpp"
#include "util/static_graph.hpp"

#include <boost/filesystem/fstream.hpp>
#include <boost/iostreams/seek.hpp>
Expand All @@ -32,6 +27,16 @@ class FileReader
boost::filesystem::ifstream input_stream;

public:
class LineWrapper : public std::string
{
friend std::istream &operator>>(std::istream &is, LineWrapper &line)
{
return std::getline(is, line);
}
};
auto GetLineIteratorBegin() { return std::istream_iterator<LineWrapper>(input_stream); }
auto GetLineIteratorEnd() { return std::istream_iterator<LineWrapper>(); }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm can this be a istreambuf_iterator? I'm not sure about this but I think these iterators work on formatted streams and streambuf works on a sequence of chars.


enum FingerprintFlag
{
VerifyFingerprint,
Expand Down Expand Up @@ -67,6 +72,7 @@ class FileReader
return;

const auto &result = input_stream.read(reinterpret_cast<char *>(dest), count * sizeof(T));

if (!result)
{
if (result.eof())
Expand All @@ -78,6 +84,11 @@ class FileReader
}
}

template <typename T> void ReadInto(std::vector<T> &target)
{
ReadInto(target.data(), target.size());
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should there be a similar function for strings? See comment above.

(Or even more generic)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is GetLine(), but the signature is a little different....

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Going to ticket for later PR


template <typename T> void ReadInto(T &target) { ReadInto(&target, 1); }

template <typename T> T ReadOne()
Expand Down Expand Up @@ -142,144 +153,23 @@ class FileReader
}
return result;
}
};

// To make function calls consistent, this function returns the fixed number of properties
inline std::size_t readPropertiesCount() { return 1; }

struct HSGRHeader
{
std::uint32_t checksum;
std::uint64_t number_of_nodes;
std::uint64_t number_of_edges;
};

// Reads the checksum, number of nodes and number of edges written in the header file of a `.hsgr`
// file and returns them in a HSGRHeader struct
inline HSGRHeader readHSGRHeader(io::FileReader &input_file)
{
const util::FingerPrint fingerprint_valid = util::FingerPrint::GetValid();
const auto fingerprint_loaded = input_file.ReadOne<util::FingerPrint>();
if (!fingerprint_loaded.TestGraphUtil(fingerprint_valid))
{
util::SimpleLogger().Write(logWARNING) << ".hsgr was prepared with different build.\n"
"Reprocess to get rid of this warning.";
}

HSGRHeader header;
input_file.ReadInto(header.checksum);
input_file.ReadInto(header.number_of_nodes);
input_file.ReadInto(header.number_of_edges);

BOOST_ASSERT_MSG(0 != header.number_of_nodes, "number of nodes is zero");
// number of edges can be zero, this is the case in a few test fixtures

return header;
}

// Reads the graph data of a `.hsgr` file into memory
// Needs to be called after readHSGRHeader() to get the correct offset in the stream
using NodeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::NodeArrayEntry;
using EdgeT = typename util::StaticGraph<contractor::QueryEdge::EdgeData>::EdgeArrayEntry;
inline void readHSGR(io::FileReader &input_file,
NodeT *node_buffer,
const std::uint64_t number_of_nodes,
EdgeT *edge_buffer,
const std::uint64_t number_of_edges)
{
BOOST_ASSERT(node_buffer);
BOOST_ASSERT(edge_buffer);
input_file.ReadInto(node_buffer, number_of_nodes);
input_file.ReadInto(edge_buffer, number_of_edges);
}

// Loads datasource_indexes from .datasource_indexes into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readDatasourceIndexes(io::FileReader &datasource_indexes_file,
uint8_t *datasource_buffer,
const std::uint64_t number_of_datasource_indexes)
{
BOOST_ASSERT(datasource_buffer);
datasource_indexes_file.ReadInto(datasource_buffer, number_of_datasource_indexes);
}

// Loads edge data from .edge files into memory which includes its
// geometry, name ID, turn instruction, lane data ID, travel mode, entry class ID
// Needs to be called after readElementCount() to get the correct offset in the stream
inline void readEdges(io::FileReader &edges_input_file,
GeometryID *geometry_list,
NameID *name_id_list,
extractor::guidance::TurnInstruction *turn_instruction_list,
LaneDataID *lane_data_id_list,
extractor::TravelMode *travel_mode_list,
EntryClassID *entry_class_id_list,
util::guidance::TurnBearing *pre_turn_bearing_list,
util::guidance::TurnBearing *post_turn_bearing_list,
const std::uint64_t number_of_edges)
{
BOOST_ASSERT(geometry_list);
BOOST_ASSERT(name_id_list);
BOOST_ASSERT(turn_instruction_list);
BOOST_ASSERT(lane_data_id_list);
BOOST_ASSERT(travel_mode_list);
BOOST_ASSERT(entry_class_id_list);
extractor::OriginalEdgeData current_edge_data;
for (std::uint64_t i = 0; i < number_of_edges; ++i)
{
edges_input_file.ReadInto(current_edge_data);

geometry_list[i] = current_edge_data.via_geometry;
name_id_list[i] = current_edge_data.name_id;
turn_instruction_list[i] = current_edge_data.turn_instruction;
lane_data_id_list[i] = current_edge_data.lane_data_id;
travel_mode_list[i] = current_edge_data.travel_mode;
entry_class_id_list[i] = current_edge_data.entry_classid;
pre_turn_bearing_list[i] = current_edge_data.pre_turn_bearing;
post_turn_bearing_list[i] = current_edge_data.post_turn_bearing;
}
}

// Loads coordinates and OSM node IDs from .nodes files into memory
// Needs to be called after readElementCount() to get the correct offset in the stream
template <typename OSMNodeIDVectorT>
void readNodes(io::FileReader &nodes_file,
util::Coordinate *coordinate_list,
OSMNodeIDVectorT &osmnodeid_list,
const std::uint64_t number_of_coordinates)
{
BOOST_ASSERT(coordinate_list);
extractor::QueryNode current_node;
for (std::uint64_t i = 0; i < number_of_coordinates; ++i)
std::string ReadLine()
{
nodes_file.ReadInto(current_node);
coordinate_list[i] = util::Coordinate(current_node.lon, current_node.lat);
osmnodeid_list.push_back(current_node.node_id);
BOOST_ASSERT(coordinate_list[i].IsValid());
std::string thisline;
try
{
std::getline(input_stream, thisline);
}
catch (const std::ios_base::failure &e)
{
// EOF is OK here, everything else, re-throw
if (!input_stream.eof())
throw;
}
return thisline;
}
}

// Reads datasource names out of .datasource_names files and metadata such as
// the length and offset of each name
struct DatasourceNamesData
{
std::vector<char> names;
std::vector<std::size_t> offsets;
std::vector<std::size_t> lengths;
};
inline DatasourceNamesData readDatasourceNames(io::FileReader &datasource_names_file)
{
DatasourceNamesData datasource_names_data;
std::vector<std::string> lines = datasource_names_file.ReadLines();
for (const auto &name : lines)
{
datasource_names_data.offsets.push_back(datasource_names_data.names.size());
datasource_names_data.lengths.push_back(name.size());
std::copy(name.c_str(),
name.c_str() + name.size(),
std::back_inserter(datasource_names_data.names));
}
return datasource_names_data;
}
}
}
}
Expand Down
Loading