Csv disk index

SConstruct

@@ -400,6 +400,7 @@ opts.AddVariables(
     BoolVariable('DEMO', 'Compile demo c++ application', 'True'),
     BoolVariable('PGSQL2SQLITE', 'Compile and install a utility to convert postgres tables to sqlite', 'False'),
     BoolVariable('SHAPEINDEX', 'Compile and install a utility to generate shapefile indexes in the custom format (.index) Mapnik supports', 'True'),
+    BoolVariable('CSVINDEX', 'Compile and install a utility to generate CSV file indexes in the custom format (.index) Mapnik supports', 'True'),
     BoolVariable('SVG2PNG', 'Compile and install a utility to generate render an svg file to a png on the command line', 'False'),
     BoolVariable('NIK2IMG', 'Compile and install a utility to generate render a map to an image', 'True'),
     BoolVariable('COLOR_PRINT', 'Print build status information in color', 'True'),
@@ -1963,6 +1964,8 @@ if not HELP_REQUESTED:
         if 'boost_program_options%s' % env['BOOST_APPEND'] in env['LIBS']:
             if env['SHAPEINDEX']:
                 SConscript('utils/shapeindex/build.py')
+            if env['CSVINDEX']:
+                SConscript('utils/csvindex/build.py')
             # Build the pgsql2psqlite app if requested
             if env['PGSQL2SQLITE']:
                 SConscript('utils/pgsql2sqlite/build.py')

include/mapnik/quad_tree.hpp

@@ -30,20 +30,21 @@
 // stl
 #include <algorithm>
 #include <vector>
+#include <type_traits>
 
 namespace mapnik
 {
 template <typename T>
 class quad_tree : util::noncopyable
 {
+    using value_type = T;
     struct node
     {
-        using value_t = T;
-        using cont_t = std::vector<T>;
-        using iterator = typename cont_t::iterator;
-        using const_iterator = typename cont_t::const_iterator;
+        using cont_type = std::vector<T>;
+        using iterator = typename cont_type::iterator;
+        using const_iterator = typename cont_type::const_iterator;
         box2d<double> extent_;
-        cont_t cont_;
+        cont_type cont_;
         node * children_[4];
 
         explicit node(box2d<double> const& ext)
@@ -76,18 +77,28 @@ class quad_tree : util::noncopyable
         {
             return cont_.end();
         }
+
+        int num_subnodes() const
+        {
+            int count = 0;
+            for (int i = 0; i < 4; ++i)
+            {
+                if (children_[i]) ++count;
+            }
+            return count;
+        }
         ~node () {}
     };
 
-    using nodes_t = std::vector<std::unique_ptr<node> >;
-    using cont_t = typename node::cont_t;
-    using node_data_iterator = typename cont_t::iterator;
+    using nodes_type = std::vector<std::unique_ptr<node> >;
+    using cont_type = typename node::cont_type;
+    using node_data_iterator = typename cont_type::iterator;
 
 public:
-    using iterator = typename nodes_t::iterator;
-    using const_iterator = typename nodes_t::const_iterator;
-    using result_t = typename std::vector<std::reference_wrapper<T> >;
-    using query_iterator = typename result_t::iterator;
+    using iterator = typename nodes_type::iterator;
+    using const_iterator = typename nodes_type::const_iterator;
+    using result_type = typename std::vector<std::reference_wrapper<T> >;
+    using query_iterator = typename result_type::iterator;
 
     explicit quad_tree(box2d<double> const& ext,
                        unsigned int max_depth = 8,
@@ -143,9 +154,41 @@ class quad_tree : util::noncopyable
         return root_->extent_;
     }
 
+    int count() const
+    {
+        return count_nodes(root_);
+    }
+
+    int count_items() const
+    {
+        int count = 0;
+        count_items(root_, count);
+        return count;
+    }
+    void trim()
+    {
+        trim_tree(root_);
+    }
+
+    template <typename OutputStream>
+    void write(OutputStream & out)
+    {
+        static_assert(std::is_standard_layout<value_type>::value,
+                      "Values stored in quad-tree must be standard layout types to allow serialisation");
+        char header[16];
+        std::memset(header,0,16);
+        header[0]='m';
+        header[1]='a';
+        header[2]='p';
+        header[3]='n';
+        header[4]='i';
+        header[5]='k';
+        out.write(header,16);
+        write_node(out,root_);
+    }
 private:
 
-    void query_node(box2d<double> const& box, result_t & result, node * node_) const
+    void query_node(box2d<double> const& box, result_type & result, node * node_) const
     {
         if (node_)
         {
@@ -208,10 +251,107 @@ class quad_tree : util::noncopyable
         ext[3]=box2d<double>(hix - width * ratio_,hiy - height*ratio_,hix,hiy);
     }
 
+    void trim_tree(node * n)
+    {
+        if (n)
+        {
+            for (int i = 0; i < 4; ++i)
+            {
+                trim_tree(n->children_[i]);
+            }
+            if (n->num_subnodes() == 1 && n->cont_.size() == 0)
+            {
+                for (int i = 0; i < 4; ++i)
+                {
+                    if (n->children_[i])
+                    {
+                        n = n->children_[i];
+                        break;
+                    }
+                }
+            }
+        }
+
+    }
+
+    int count_nodes(node const* n) const
+    {
+        if (!n) return 0;
+        else
+        {
+            int count = 1;
+            for (int i = 0; i < 4; ++i)
+            {
+                count += count_nodes(n->children_[i]);
+            }
+            return count;
+        }
+    }
+
+    void count_items(node const* n,int& count) const
+    {
+        if (n)
+        {
+            count += n->cont_.size();
+            for (int i = 0; i < 4; ++i)
+            {
+                count_items(n->children_[i],count);
+            }
+        }
+    }
+
+    int subnode_offset(node const* n) const
+    {
+        int offset = 0;
+        for (int i = 0; i < 4; i++)
+        {
+            if (n->children_[i])
+            {
+                offset +=sizeof(box2d<double>) + (n->children_[i]->cont_.size() * sizeof(value_type)) + 3 * sizeof(int);
+                offset +=subnode_offset(n->children_[i]);
+            }
+        }
+        return offset;
+    }
+
+    template <typename OutputStream>
+    void write_node(OutputStream & out, node const* n) const
+    {
+        if (n)
+        {
+            int offset=subnode_offset(n);
+            int shape_count=n->cont_.size();
+            int recsize=sizeof(box2d<double>) + 3 * sizeof(int) + shape_count * sizeof(value_type);
+            std::unique_ptr<char[]> node_record(new char[recsize]);
+            std::memset(node_record.get(), 0, recsize);
+            std::memcpy(node_record.get(), &offset, 4);
+            std::memcpy(node_record.get() + 4, &n->extent_, sizeof(box2d<double>));
+            std::memcpy(node_record.get() + 36, &shape_count, 4);
+            for (int i=0; i < shape_count; ++i)
+            {
+                memcpy(node_record.get() + 40 + i * sizeof(value_type), &(n->cont_[i]),sizeof(value_type));
+            }
+            int num_subnodes=0;
+            for (int i = 0; i < 4; ++i)
+            {
+                if (n->children_[i])
+                {
+                    ++num_subnodes;
+                }
+            }
+            std::memcpy(node_record.get() + 40 + shape_count * sizeof(value_type),&num_subnodes,4);
+            out.write(node_record.get(),recsize);
+            for (int i = 0; i < 4; ++i)
+            {
+                write_node(out, n->children_[i]);
+            }
+        }
+    }
+
     const unsigned int max_depth_;
     const double ratio_;
-    result_t query_result_;
-    nodes_t nodes_;
+    result_type query_result_;
+    nodes_type nodes_;
     node * root_;
 
 };

include/mapnik/util/spatial_index.hpp

@@ -0,0 +1,118 @@
+/*****************************************************************************
+ *
+ * This file is part of Mapnik (c++ mapping toolkit)
+ *
+ * Copyright (C) 2015 Artem Pavlenko
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ *****************************************************************************/
+
+#ifndef MAPNIK_UTIL_SPATIAL_INDEX_HPP
+#define MAPNIK_UTIL_SPATIAL_INDEX_HPP
+
+//mapnik
+#include <mapnik/coord.hpp>
+#include <mapnik/box2d.hpp>
+#include <mapnik/query.hpp>
+#include <mapnik/geom_util.hpp>
+// stl
+#include <type_traits>
+
+using mapnik::box2d;
+using mapnik::query;
+
+namespace mapnik { namespace util {
+
+template <typename Value, typename Filter, typename InputStream>
+class spatial_index
+{
+public:
+    static void query(Filter const& filter, InputStream& in,std::vector<Value>& pos);
+    static box2d<double> bounding_box( InputStream& in );
+private:
+
+    spatial_index();
+    ~spatial_index();
+    spatial_index(spatial_index const&);
+    spatial_index& operator=(spatial_index const&);
+    static int read_ndr_integer(InputStream& in);
+    static void read_envelope(InputStream& in, box2d<double>& envelope);
+    static void query_node(Filter const& filter, InputStream& in, std::vector<Value> & results);
+};
+
+template <typename Value, typename Filter, typename InputStream>
+box2d<double> spatial_index<Value, Filter, InputStream>::bounding_box(InputStream& in)
+{
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout type");
+    in.seekg(16 + 4, std::ios::beg);
+    box2d<double> box;
+    read_envelope(in, box);
+    in.seekg(0, std::ios::beg);
+    return box;
+}
+
+template <typename Value, typename Filter, typename InputStream>
+void spatial_index<Value, Filter, InputStream>::query(Filter const& filter, InputStream& in, std::vector<Value>& results)
+{
+    static_assert(std::is_standard_layout<Value>::value, "Values stored in quad-tree must be standard layout types");
+    in.seekg(16, std::ios::beg);
+    query_node(filter, in, results);
+}
+
+template <typename Value, typename Filter, typename InputStream>
+void spatial_index<Value, Filter, InputStream>::query_node(Filter const& filter, InputStream& in, std::vector<Value>& results)
+{
+    int offset = read_ndr_integer(in);
+    box2d<double> node_ext;
+    read_envelope(in, node_ext);
+    int num_shapes = read_ndr_integer(in);
+    if (!filter.pass(node_ext))
+    {
+        in.seekg(offset + num_shapes * sizeof(Value) + 4, std::ios::cur);
+        return;
+    }
+
+    for (int i = 0; i < num_shapes; ++i)
+    {
+        Value item;
+        in.read(reinterpret_cast<char*>(&item), sizeof(Value));
+        results.push_back(std::move(item));
+    }
+
+    int children = read_ndr_integer(in);
+    for (int j = 0; j < children; ++j)
+    {
+        query_node(filter, in, results);
+    }
+}
+
+template <typename Value, typename Filter, typename InputStream>
+int spatial_index<Value, Filter, InputStream>::read_ndr_integer(InputStream& in)
+{
+    char b[4];
+    in.read(b, 4);
+    return (b[0] & 0xff) | (b[1] & 0xff) << 8 | (b[2] & 0xff) << 16 | (b[3] & 0xff) << 24;
+}
+
+template <typename Value, typename Filter, typename InputStream>
+void spatial_index<Value, Filter, InputStream>::read_envelope(InputStream& in, box2d<double>& envelope)
+{
+    in.read(reinterpret_cast<char*>(&envelope), sizeof(envelope));
+}
+
+}} // mapnik/util
+
+#endif // MAPNIK_UTIL_SPATIAL_INDEX_HPP

plugins/input/csv/build.py

@@ -42,6 +42,7 @@
         %(PLUGIN_NAME)s_datasource.cpp
         %(PLUGIN_NAME)s_featureset.cpp
         %(PLUGIN_NAME)s_inline_featureset.cpp
+        %(PLUGIN_NAME)s_index_featureset.cpp
         """ % locals()
     )