Add CRC checksums to EBG and turns data

CMakeLists.txt

@@ -679,7 +679,8 @@ set(UTIL_LIBRARIES
     ${CMAKE_THREAD_LIBS_INIT}
     ${MAYBE_STXXL_LIBRARY}
     ${TBB_LIBRARIES}
-    ${MAYBE_COVERAGE_LIBRARIES})
+    ${MAYBE_COVERAGE_LIBRARIES}
+    ${ZLIB_LIBRARY})
 
 # Libraries
 target_link_libraries(osrm ${ENGINE_LIBRARIES})

include/contractor/files.hpp

@@ -19,7 +19,8 @@ template <typename QueryGraphT, typename EdgeFilterT>
 inline void readGraph(const boost::filesystem::path &path,
                       unsigned &checksum,
                       QueryGraphT &graph,
-                      std::vector<EdgeFilterT> &edge_filter)
+                      std::vector<EdgeFilterT> &edge_filter,
+                      std::uint32_t &connectivity_checksum)
 {
     static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
                       std::is_same<QueryGraph, QueryGraphT>::value,
@@ -39,14 +40,16 @@ inline void readGraph(const boost::filesystem::path &path,
     {
         storage::serialization::read(reader, edge_filter[index]);
     }
+    reader.ReadInto(connectivity_checksum);
 }
 
 // writes .osrm.hsgr file
 template <typename QueryGraphT, typename EdgeFilterT>
 inline void writeGraph(const boost::filesystem::path &path,
                        unsigned checksum,
                        const QueryGraphT &graph,
-                       const std::vector<EdgeFilterT> &edge_filter)
+                       const std::vector<EdgeFilterT> &edge_filter,
+                       const std::uint32_t connectivity_checksum)
 {
     static_assert(std::is_same<QueryGraphView, QueryGraphT>::value ||
                       std::is_same<QueryGraph, QueryGraphT>::value,
@@ -64,6 +67,7 @@ inline void writeGraph(const boost::filesystem::path &path,
     {
         storage::serialization::write(writer, filter);
     }
+    writer.WriteOne(connectivity_checksum);
 }
 }
 }

include/extractor/edge_based_graph_factory.hpp

@@ -92,6 +92,7 @@ class EdgeBasedGraphFactory
     void GetEdgeBasedNodeSegments(std::vector<EdgeBasedNodeSegment> &nodes);
     void GetStartPointMarkers(std::vector<bool> &node_is_startpoint);
     void GetEdgeBasedNodeWeights(std::vector<EdgeWeight> &output_node_weights);
+    std::uint32_t GetConnectivityChecksum() const;
 
     std::uint64_t GetNumberOfEdgeBasedNodes() const;
 
@@ -133,6 +134,7 @@ class EdgeBasedGraphFactory
     std::vector<EdgeBasedNodeSegment> m_edge_based_node_segments;
     EdgeBasedNodeDataContainer &m_edge_based_node_container;
     util::DeallocatingVector<EdgeBasedEdge> m_edge_based_edge_list;
+    std::uint32_t m_connectivity_checksum;
 
     // The number of edge-based nodes is mostly made up out of the edges in the node-based graph.
     // Any edge in the node-based graph represents a node in the edge-based graph. In addition, we

include/extractor/extractor.hpp

@@ -84,7 +84,8 @@ class Extractor
         std::vector<EdgeBasedNodeSegment> &edge_based_node_segments,
         std::vector<bool> &node_is_startpoint,
         std::vector<EdgeWeight> &edge_based_node_weights,
-        util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list);
+        util::DeallocatingVector<EdgeBasedEdge> &edge_based_edge_list,
+        std::uint32_t &connectivity_checksum);
 
     void FindComponents(unsigned max_edge_id,
                         const util::DeallocatingVector<EdgeBasedEdge> &input_edge_list,

include/extractor/files.hpp

@@ -78,27 +78,31 @@ inline void writeProfileProperties(const boost::filesystem::path &path,
 template <typename EdgeBasedEdgeVector>
 void writeEdgeBasedGraph(const boost::filesystem::path &path,
                          EdgeID const number_of_edge_based_nodes,
-                         const EdgeBasedEdgeVector &edge_based_edge_list)
+                         const EdgeBasedEdgeVector &edge_based_edge_list,
+                         const std::uint32_t connectivity_checksum)
 {
     static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
 
     storage::io::FileWriter writer(path, storage::io::FileWriter::GenerateFingerprint);
 
     writer.WriteElementCount64(number_of_edge_based_nodes);
     storage::serialization::write(writer, edge_based_edge_list);
+    writer.WriteOne(connectivity_checksum);
 }
 
 template <typename EdgeBasedEdgeVector>
 void readEdgeBasedGraph(const boost::filesystem::path &path,
                         EdgeID &number_of_edge_based_nodes,
-                        EdgeBasedEdgeVector &edge_based_edge_list)
+                        EdgeBasedEdgeVector &edge_based_edge_list,
+                        std::uint32_t &connectivity_checksum)
 {
     static_assert(std::is_same<typename EdgeBasedEdgeVector::value_type, EdgeBasedEdge>::value, "");
 
     storage::io::FileReader reader(path, storage::io::FileReader::VerifyFingerprint);
 
     number_of_edge_based_nodes = reader.ReadElementCount64();
     storage::serialization::read(reader, edge_based_edge_list);
+    reader.ReadInto(connectivity_checksum);
 }
 
 // reads .osrm.nodes

include/guidance/files.hpp

@@ -19,7 +19,9 @@ namespace files
 
 // reads .osrm.edges
 template <typename TurnDataT>
-inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_data)
+inline void readTurnData(const boost::filesystem::path &path,
+                         TurnDataT &turn_data,
+                         std::uint32_t &connectivity_checksum)
 {
     static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
                       std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@@ -28,12 +30,14 @@ inline void readTurnData(const boost::filesystem::path &path, TurnDataT &turn_da
     const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, turn_data);
+    serialization::read(reader, turn_data, connectivity_checksum);
 }
 
 // writes .osrm.edges
 template <typename TurnDataT>
-inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &turn_data)
+inline void writeTurnData(const boost::filesystem::path &path,
+                          const TurnDataT &turn_data,
+                          const std::uint32_t connectivity_checksum)
 {
     static_assert(std::is_same<guidance::TurnDataContainer, TurnDataT>::value ||
                       std::is_same<guidance::TurnDataView, TurnDataT>::value ||
@@ -42,7 +46,7 @@ inline void writeTurnData(const boost::filesystem::path &path, const TurnDataT &
     const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, turn_data);
+    serialization::write(writer, turn_data, connectivity_checksum);
 }
 }
 }

include/guidance/guidance_processing.hpp

@@ -41,7 +41,8 @@ void annotateTurns(const util::NodeBasedDynamicGraph &node_based_graph,
                    guidance::TurnDataExternalContainer &turn_data_container,
                    BearingClassesVector &bearing_class_by_node_based_node,
                    BearingClassesMap &bearing_class_hash,
-                   EntryClassesMap &entry_class_hash);
+                   EntryClassesMap &entry_class_hash,
+                   std::uint32_t &connectivity_checksum);
 
 } // namespace customizer
 } // namespace osrm

include/guidance/serialization.hpp

@@ -18,24 +18,28 @@ namespace serialization
 // read/write for turn data file
 template <storage::Ownership Ownership>
 inline void read(storage::io::FileReader &reader,
-                 guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
+                 guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
+                 std::uint32_t &connectivity_checksum)
 {
     storage::serialization::read(reader, turn_data_container.turn_instructions);
     storage::serialization::read(reader, turn_data_container.lane_data_ids);
     storage::serialization::read(reader, turn_data_container.entry_class_ids);
     storage::serialization::read(reader, turn_data_container.pre_turn_bearings);
     storage::serialization::read(reader, turn_data_container.post_turn_bearings);
+    reader.ReadInto(connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>
 inline void write(storage::io::FileWriter &writer,
-                  const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container)
+                  const guidance::detail::TurnDataContainerImpl<Ownership> &turn_data_container,
+                  const std::uint32_t connectivity_checksum)
 {
     storage::serialization::write(writer, turn_data_container.turn_instructions);
     storage::serialization::write(writer, turn_data_container.lane_data_ids);
     storage::serialization::write(writer, turn_data_container.entry_class_ids);
     storage::serialization::write(writer, turn_data_container.pre_turn_bearings);
     storage::serialization::write(writer, turn_data_container.post_turn_bearings);
+    writer.WriteOne(connectivity_checksum);
 }
 }
 }

include/guidance/turn_data_container.hpp

@@ -24,11 +24,14 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl;
 namespace serialization
 {
 template <storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, detail::TurnDataContainerImpl<Ownership> &turn_data);
+void read(storage::io::FileReader &reader,
+          detail::TurnDataContainerImpl<Ownership> &turn_data,
+          std::uint32_t &connectivity_checksum);
 
 template <storage::Ownership Ownership>
 void write(storage::io::FileWriter &writer,
-           const detail::TurnDataContainerImpl<Ownership> &turn_data);
+           const detail::TurnDataContainerImpl<Ownership> &turn_data,
+           const std::uint32_t connectivity_checksum);
 }
 
 struct TurnData
@@ -95,9 +98,11 @@ template <storage::Ownership Ownership> class TurnDataContainerImpl
     }
 
     friend void serialization::read<Ownership>(storage::io::FileReader &reader,
-                                               TurnDataContainerImpl &turn_data_container);
+                                               TurnDataContainerImpl &turn_data_container,
+                                               std::uint32_t &connectivity_checksum);
     friend void serialization::write<Ownership>(storage::io::FileWriter &writer,
-                                                const TurnDataContainerImpl &turn_data_container);
+                                                const TurnDataContainerImpl &turn_data_container,
+                                                const std::uint32_t connectivity_checksum);
 
   private:
     Vector<TurnInstruction> turn_instructions;

include/partition/edge_based_graph.hpp

@@ -36,6 +36,16 @@ struct DynamicEdgeBasedGraph : util::DynamicGraph<EdgeBasedGraphEdgeData>
 {
     using Base = util::DynamicGraph<EdgeBasedGraphEdgeData>;
     using Base::Base;
+
+    template <class ContainerT>
+    DynamicEdgeBasedGraph(const NodeIterator nodes,
+                          const ContainerT &graph,
+                          std::uint32_t connectivity_checksum)
+        : Base(nodes, graph), connectivity_checksum(connectivity_checksum)
+    {
+    }
+
+    std::uint32_t connectivity_checksum;
 };
 
 struct DynamicEdgeBasedGraphEdge : DynamicEdgeBasedGraph::InputEdge

include/partition/edge_based_graph_reader.hpp

@@ -187,12 +187,13 @@ inline DynamicEdgeBasedGraph LoadEdgeBasedGraph(const boost::filesystem::path &p
 {
     EdgeID number_of_edge_based_nodes;
     std::vector<extractor::EdgeBasedEdge> edges;
-    extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges);
+    std::uint32_t checksum;
+    extractor::files::readEdgeBasedGraph(path, number_of_edge_based_nodes, edges, checksum);
 
     auto directed = splitBidirectionalEdges(edges);
     auto tidied = prepareEdgesForUsageInGraph<DynamicEdgeBasedGraphEdge>(std::move(directed));
 
-    return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied));
+    return DynamicEdgeBasedGraph(number_of_edge_based_nodes, std::move(tidied), checksum);
 }
 
 } // ns partition

include/partition/files.hpp

@@ -16,7 +16,9 @@ namespace files
 
 // reads .osrm.mldgr file
 template <typename MultiLevelGraphT>
-inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &graph)
+inline void readGraph(const boost::filesystem::path &path,
+                      MultiLevelGraphT &graph,
+                      std::uint32_t &connectivity_checksum)
 {
     static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@@ -25,12 +27,14 @@ inline void readGraph(const boost::filesystem::path &path, MultiLevelGraphT &gra
     const auto fingerprint = storage::io::FileReader::VerifyFingerprint;
     storage::io::FileReader reader{path, fingerprint};
 
-    serialization::read(reader, graph);
+    serialization::read(reader, graph, connectivity_checksum);
 }
 
 // writes .osrm.mldgr file
 template <typename MultiLevelGraphT>
-inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGraphT &graph)
+inline void writeGraph(const boost::filesystem::path &path,
+                       const MultiLevelGraphT &graph,
+                       const std::uint32_t connectivity_checksum)
 {
     static_assert(std::is_same<customizer::MultiLevelEdgeBasedGraphView, MultiLevelGraphT>::value ||
                       std::is_same<customizer::MultiLevelEdgeBasedGraph, MultiLevelGraphT>::value,
@@ -39,7 +43,7 @@ inline void writeGraph(const boost::filesystem::path &path, const MultiLevelGrap
     const auto fingerprint = storage::io::FileWriter::GenerateFingerprint;
     storage::io::FileWriter writer{path, fingerprint};
 
-    serialization::write(writer, graph);
+    serialization::write(writer, graph, connectivity_checksum);
 }
 
 // read .osrm.partition file

include/partition/multi_level_graph.hpp

@@ -24,10 +24,14 @@ template <typename EdgeDataT, storage::Ownership Ownership> class MultiLevelGrap
 namespace serialization
 {
 template <typename EdgeDataT, storage::Ownership Ownership>
-void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph);
+void read(storage::io::FileReader &reader,
+          MultiLevelGraph<EdgeDataT, Ownership> &graph,
+          std::uint32_t &connectivity_checksum);
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-void write(storage::io::FileWriter &writer, const MultiLevelGraph<EdgeDataT, Ownership> &graph);
+void write(storage::io::FileWriter &writer,
+           const MultiLevelGraph<EdgeDataT, Ownership> &graph,
+           const std::uint32_t connectivity_checksum);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
@@ -199,12 +203,15 @@ class MultiLevelGraph : public util::StaticGraph<EdgeDataT, Ownership>
 
     friend void
     serialization::read<EdgeDataT, Ownership>(storage::io::FileReader &reader,
-                                              MultiLevelGraph<EdgeDataT, Ownership> &graph);
+                                              MultiLevelGraph<EdgeDataT, Ownership> &graph,
+                                              std::uint32_t &connectivity_checksum);
     friend void
     serialization::write<EdgeDataT, Ownership>(storage::io::FileWriter &writer,
-                                               const MultiLevelGraph<EdgeDataT, Ownership> &graph);
+                                               const MultiLevelGraph<EdgeDataT, Ownership> &graph,
+                                               const std::uint32_t connectivity_checksum);
 
     Vector<EdgeOffset> node_to_edge_offset;
+    std::uint32_t connectivity_checksum;
 };
 }
 }

include/partition/serialization.hpp

@@ -18,20 +18,25 @@ namespace serialization
 {
 
 template <typename EdgeDataT, storage::Ownership Ownership>
-inline void read(storage::io::FileReader &reader, MultiLevelGraph<EdgeDataT, Ownership> &graph)
+inline void read(storage::io::FileReader &reader,
+                 MultiLevelGraph<EdgeDataT, Ownership> &graph,
+                 std::uint32_t &connectivity_checksum)
 {
     storage::serialization::read(reader, graph.node_array);
     storage::serialization::read(reader, graph.edge_array);
     storage::serialization::read(reader, graph.node_to_edge_offset);
+    reader.ReadInto(connectivity_checksum);
 }
 
 template <typename EdgeDataT, storage::Ownership Ownership>
 inline void write(storage::io::FileWriter &writer,
-                  const MultiLevelGraph<EdgeDataT, Ownership> &graph)
+                  const MultiLevelGraph<EdgeDataT, Ownership> &graph,
+                  const std::uint32_t connectivity_checksum)
 {
     storage::serialization::write(writer, graph.node_array);
     storage::serialization::write(writer, graph.edge_array);
     storage::serialization::write(writer, graph.node_to_edge_offset);
+    writer.WriteOne(connectivity_checksum);
 }
 
 template <storage::Ownership Ownership>

include/updater/updater.hpp

@@ -17,12 +17,14 @@ class Updater
   public:
     Updater(UpdaterConfig config_) : config(std::move(config_)) {}
 
-    using NumNodesAndEdges = std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>>;
+    using NumNodesAndEdges =
+        std::tuple<EdgeID, std::vector<extractor::EdgeBasedEdge>, std::uint32_t>;
     NumNodesAndEdges LoadAndUpdateEdgeExpandedGraph() const;
 
     EdgeID
     LoadAndUpdateEdgeExpandedGraph(std::vector<extractor::EdgeBasedEdge> &edge_based_edge_list,
-                                   std::vector<EdgeWeight> &node_weights) const;
+                                   std::vector<EdgeWeight> &node_weights,
+                                   std::uint32_t &connectivity_checksum) const;
 
   private:
     UpdaterConfig config;